import math

from mongoDB import MangoDBConnection
from YCAArticleSpider import ArticleSpider

if __name__ == '__main__':
    mongoDBClient = MangoDBConnection()
    db = mongoDBClient.myclient['water']
    directory = db['directory']
    article = db['article']

    limit = 10
    count_skip = math.ceil(directory.estimated_document_count() / limit)

    for i in range(count_skip):
        directorys = directory.find({}).limit(limit).skip((i + 1) * limit)
        # directorys = directory.find({}).limit(1).skip(3)

        urls = []
        for d in directorys:
            urls.append(d.get('url'))

        spider = ArticleSpider(source='少年中国评论')
        bson = spider.run(urls=urls)

        # print(bson)
        x = article.insert_many(bson)
